{
 "nbformat": 4,
 "nbformat_minor": 2,
 "metadata": {
  "language_info": {
   "name": "python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "version": "3.7.5"
  },
  "orig_nbformat": 2,
  "file_extension": ".py",
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "cells": [
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 损失实现\n",
    "\n",
    "交叉熵损失和二值逻辑损失实现(numpy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### 交叉熵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy(output, target):\n",
    "    out_exp = np.exp(output)\n",
    "    out_cls = np.array([out_exp[i, t] for i, t in enumerate(target)])\n",
    "    ce = -np.log(out_cls / out_exp.sum(1))\n",
    "    return ce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy(output, target, weight=None, reduction='mean'):\n",
    "    out_exp = np.exp(output)\n",
    "    out_cls = np.array([out_exp[i, t] for i, t in enumerate(target)])\n",
    "    ce = -np.log(out_cls / out_exp.sum(1))\n",
    "    if weight is not None: # 加上样本权重\n",
    "        weight = np.array(weight)\n",
    "        ce = ce * weight\n",
    "    else:\n",
    "        weight = np.ones_like(ce)\n",
    "        \n",
    "    if reduction == 'mean': # 加权平均\n",
    "        ce = (ce / weight.sum()).sum()\n",
    "    elif reduction == 'sum':\n",
    "        ce = ce.sum()\n",
    "    return ce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "0.6897266409702164\n"
    }
   ],
   "source": [
    "# output = np.array([[0.1, 0.8, 0.1], [0.5, 0.1, 0.9]])\n",
    "# target = np.array([1, 2])\n",
    "\n",
    "output = np.array([[0.1, 0.8, 0.1]])\n",
    "target = np.array([1])\n",
    "\n",
    "print(cross_entropy(output, target))\n",
    "# 二分类交叉熵不过是上面的特例而已"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### 二值逻辑熵\n",
    "\n",
    "$$\n",
    "\\ell(x, y) = L = \\{l_1,\\dots,l_N\\}^\\top, \\quad\n",
    "        l_n = - w_n \\left[ y_n \\cdot \\log \\sigma(x_n)\n",
    "        + (1 - y_n) \\cdot \\log (1 - \\sigma(x_n)) \\right]\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bce_with_logits(output, target):\n",
    "    out_sigmoid = 1/(1+np.exp(-output))\n",
    "    bce = -(target*np.log(out_sigmoid) + (1-target)*np.log(1-out_sigmoid)).mean(1)\n",
    "    return bce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "array([0.81996466, 0.81987584])"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output = np.array([[0.1, 0.8, 0.1], [0.5, 0.1, 0.9]])\n",
    "target = np.array([[1, 0, 1],[1, 0, 0]])\n",
    "\n",
    "bce_with_logits(output, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}